package util;

import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/*这个类作用是处理采集下的网页内容，使用了正则表达式截取出需要采集的网址*/

public class SpiderYouDaoRobot implements Runnable {

	public void run() {
		// 两次采集搜索引擎之前的时间间隔
		final int minIntervalTimer = 10000;
		// 程序出错后重启的时间间隔
		final long errWating = 20000;
		// 搜索引擎的编号
		final int searchNumber = 2;
		Connection conn = null;
		Statement stmt = null;
		Connection connCheck = null;
		Statement stmtCheck = null;
		Connection connGetSite = null;
		Statement stmtGetSite = null;
		java.sql.ResultSet rs = null;
		java.sql.ResultSet rsGetSite = null;
		// workingID是个标记位，指示在现在处理到哪一位了
		int workingID = 0;
		String keyword = null;
		while (true) {
			try {
				Class.forName("com.mysql.jdbc.Driver");
				conn = DriverManager.getConnection("jdbc:mysql://" + host + "/" + dBName + "?user=root");
				connGetSite = conn;
				connCheck = conn;
				stmtGetSite = connGetSite.createStatement();
				stmtCheck = connCheck.createStatement();
				// 查询出需要处理的网址，check是标记为0就是需要处理的
				rsGetSite = stmtGetSite.executeQuery("select * from keywords where youdaoSW = 0");
				while (rsGetSite.next()) {
					workingID = rsGetSite.getInt("id");
					String tag = rsGetSite.getString("keyword");
					keyword = URLEncoder.encode(tag, codeNo);
					// 抓取网址开始的时间点
					Long btime = System.currentTimeMillis();
					SpiderBodyXMLBing sbx = new SpiderBodyXMLBing();
					sbx.setInterval(interval);
					sbx.setRepeatTimes(repeatTimes);
					sbx.SpiderPageToXML(first + keyword + end);
					String SitesString = sbx.getWebSiteBodyString();
					sbx.clear();
					sbx = null;
					// 到这里已经处理完了当下的keywords，需要修改flter字段
					stmt = conn.createStatement();
					// 修改标记放在处理抓取下来的字符串之前目的就是如果这条数据出错了，下次在重启时也不需要再处理了，虽然少处理一天数据但保证了程序的连续性
					stmtCheck.execute("update keywords set youdaoSW = 1 where id = " + workingID);
					// 如果是0表示链接有错误返回的只是默认值
					if (SitesString == "0") {
						// 出错的没有成功抓取的连接放入badurl表
						stmt.execute("insert into badurl (id,badid,search) values (0," + workingID + "," + searchNumber + ")");

					} else {
						// 返回好页面，再抓取其中的链接
						// System.out.println(SitesString);
						Pattern p = Pattern.compile(urlPattern);
						Matcher m = p.matcher(SitesString);
						int tagURLController = 0;
						while (m.find()) {
							String url = m.group(2);
							// 过长的url处理，抓取.html .shtml等结尾前的部分，如果没有跳出这次循环这条数据不要了
							if (url.length() > 1000) {
								Pattern LongPattern = Pattern.compile("http://.*\\.shtml|http://.*\\.html|http://.*\\.htm|http://.*\\.asp|http://.*\\.aspx|http://.*\\.httm|http://.*\\.bottom|http://.*\\.xhtml|http://.*\\.shtm|http://.*\\.php|http://.*\\.jsp|http://.*\\.jhtml|http://.*\\.dhtml");
								Matcher LongMatcher = LongPattern.matcher(url);
								LongMatcher.reset();
								if (LongMatcher.find()) {
									url = LongMatcher.group();
								} else {
									continue;
								}

							}
							// 这是了重复判断机制，如果有重复的就不加入了，这就使中断后再开始采集时不会出现重复数据
							rs = stmt.executeQuery("select id from tagurl where url = '" + url + "' and tag = '" + tag + "'");
							if (!rs.next()) {
								// 再用一次正则表达式截取出仅包含域名部分的网址，正则的意义是:字母+/就是域名的结束位
								Pattern webSitePattern = Pattern.compile("[a-z]/");
								Matcher webSiteMatcher = webSitePattern.matcher(url);
								webSiteMatcher.reset();
								// 如果能够匹配出主页的网址内容插入数据库，匹配不出就不插入，匹配不出的是ip地址网页质量不会太高,continue跳过这一条
								if (webSiteMatcher.find()) {
									// 字母+/的开始位置再加2个正好就是网址的结束位置
									int j = webSiteMatcher.start() + 2;
									String netPort = new String();
									netPort = url.substring(0, j);
									stmt.execute("insert into tagurl (id,tag,url,urlnotail) values (0,'" + tag + "','" + url + "','" + netPort + "')");
								} else {
									continue;
								}
							}
							/*
							 * ————————test——————————————
							 * 测试时候用，这是一个打印输出，没抓取一个打印出一个数字可以观察程序的运行状态
							 * //tagURLController++;
							 * System.out.println(tagURLController);
							 */
						}
						// tagURLController=0也就是一个链接也没抓取到，把这个页计入到badurl表中
						if (tagURLController == 0) {
							stmt.execute("insert into badurl (id,badid,search) values (0," + workingID + "," + searchNumber + ")");
						}
					}
					System.out.println(Thread.currentThread().getName() + "-------" + (workingID));
					Long etime = System.currentTimeMillis();
					// 打印出处理使用的时间
					System.out.println(etime - btime);
					// 如果不到10秒就加一段延时
					if ((etime - btime) < minIntervalTimer) {
						Thread.sleep(minIntervalTimer - (etime - btime));
					}
				}
				flag = true;
				break;
			} catch (ClassNotFoundException e) {
				e.printStackTrace();
			} catch (UnsupportedEncodingException e) {
				e.printStackTrace();
			} catch (SQLException e) {
				// 处理
				try {
					Thread.sleep(errWating);
					continue;
				} catch (InterruptedException e1) {
					// TODO Auto-generated catch block
					e1.printStackTrace();
				}
			} catch (InterruptedException e) {
				// 处理
				try {
					Thread.sleep(errWating);
					continue;
				} catch (InterruptedException e1) {
					// TODO Auto-generated catch block
					e1.printStackTrace();
				}
			} catch (Exception e) {
				// 处理
				try {
					Thread.sleep(errWating);
					continue;
				} catch (InterruptedException e1) {
					// TODO Auto-generated catch block
					e1.printStackTrace();
				}
			} finally {
				try {
					if (rs != null) {
						rs.close();
						rs = null;
					}
					if (stmt != null) {
						stmt.close();
						stmt = null;
					}
					if (conn != null) {
						conn.close();
						conn = null;
					}
					if (rsGetSite != null) {
						rsGetSite.close();
						rsGetSite = null;
					}
					if (stmtGetSite != null) {
						stmtGetSite.close();
						stmtGetSite = null;
					}
					if (connGetSite != null) {
						connGetSite.close();
						connGetSite = null;
					}
					if (stmtCheck != null) {
						stmtCheck.close();
						stmtCheck = null;
					}
					if (connCheck != null) {
						connCheck.close();
						connCheck = null;
					}
				} catch (SQLException e) {
					e.printStackTrace();
				}
			}
		}
	}

	String urlPattern = null;
	String host = null;
	String dBName = null;
	String first = null;
	String codeNo = null;
	Boolean flag = false;
	Boolean alive = false;

	public Boolean getAlive() {
		Thread.currentThread().isAlive();
		return alive;
	}

	public Boolean getFlag() {
		return flag;
	}

	public void setCodeNo(String codeNo) {
		this.codeNo = codeNo;
	}

	public void setFirst(String first) {
		this.first = first;
	}

	String end = null;

	public void setEnd(String end) {
		this.end = end;
	}

	int interval = 0;
	int repeatTimes = 0;

	public void setRepeatTimes(int repeatTimes) {
		this.repeatTimes = repeatTimes;
	}

	public void setInterval(int interval) {
		this.interval = interval;
	}

	public void setHost(String host) {
		this.host = host;
	}

	public void setdBName(String dBName) {
		this.dBName = dBName;
	}

	public void setUrlPattern(String urlPattern) {
		this.urlPattern = urlPattern;
	}
}
